import requests
import math
import time
import redis
from lxml import etree
from threading import Thread, Lock

# 线程安全的计数器
counter_lock = Lock()
counter = 0


# 带重试机制的请求函数
def requests_get_with_retry(url):
    for _ in range(10):
        try:
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
            }
            proxyAddr = "tun-oolotr.qg.net:18447"
            authKey = "94AF792C"
            password = "79DBC0D39912"
            # 代理认证模式
            proxyUrl = f"http://{authKey}:{password}@{proxyAddr}"
            proxies = {
                "http": proxyUrl,
                "https": proxyUrl,
            }
            response = requests.get(url, headers=headers, proxies=proxies)
            html_str = response.content.decode()
            if "本次访问已触发人机验证，请按指示操作" in html_str:
                raise Exception("触发人机验证")
            if "当前系统正忙，请登录后再试" in html_str:
                raise Exception("系统繁忙")
            return response
        except Exception as e:
            with open("error.txt", "a", encoding="utf-8") as f:
                f.write(f"{url}请求失败，正在重试：{type(e)} {e}\n")
    return None


# 处理每个区域的任务函数
def task(area_name, area_url):
    global counter
    response = requests_get_with_retry(area_url)
    html_str = response.content.decode()
    root = etree.HTML(html_str)
    # 获取该区的租房数量
    house_count = int("".join(root.xpath("//span[@class='content__title--hl']/text()")))
    print(f"杭州 -- {area_name} -- {house_count}")

    # 如果租房数量小于等于3000，则处理该区域的所有页面
    if house_count <= 3000:
        for page in range(1, math.ceil(house_count / 30) + 1):
            task_url = area_url + f"pg{page}/"
            redis_conn.rpush("lianjia_task_queue", task_url)
            with counter_lock:
                counter += 1
            print(f"{area_name} -- {task_url}")
    else:
        # 如果租房数量大于3000，则按商圈进行划分
        county_list = root.xpath("//div[@id='filter']/ul[4]/li[@class='filter__item--level3  ']/a")
        for county in county_list:
            county_name = "".join(county.xpath("./text()"))
            county_url = "https://hz.lianjia.com" + "".join(county.xpath("./@href"))
            response = requests_get_with_retry(county_url)
            html_str = response.content.decode()
            root = etree.HTML(html_str)

            # 获取该商圈的租房数量
            county_house_count = int("".join(root.xpath("//span[@class='content__title--hl']/text()")))
            print(f"杭州 -- {area_name} -- {county_name} -- {county_house_count}")

            # 如果商圈租房数量小于等于3000，则处理该商圈的所有页面
            if county_house_count <= 3000:
                for page in range(1, math.ceil(county_house_count / 30) + 1):
                    task_url = county_url + f"pg{page}/"
                    redis_conn.rpush("lianjia_task_queue", task_url)
                    with counter_lock:
                        counter += 1
                    print(f"{county_name} -- {task_url}")
            else:
                # 如果商圈的租房数量大于3000，则根据价格进行划分
                price_list = [(0, 1000), (1000, 1500), (1500, 2000), (2000, 3000), (3000, 5000), (5000, 10000),
                              (10000, 10000000)]
                for price in price_list:
                    price_url = county_url + f'brp{price[0]}erp{price[1]}/'
                    response = requests_get_with_retry(price_url)
                    html_str = response.content.decode()
                    root = etree.HTML(html_str)
                    house_count = int("".join(root.xpath("//span[@class='content__title--hl']/text()")))
                    print(f"杭州 -- {area_name} -- {county_name} -- {price} -- {house_count}")

                    # 如果租房数量大于3000，则将数量设为3000
                    if house_count > 3000:
                        house_count = 3000
                    # 如果租房数量小于等于3000，则处理该价格区间的所有页面
                    if house_count <= 3000:
                        for page in range(1, math.ceil(house_count / 30) + 1):
                            task_url = county_url + f"/pg{page}/" + f'brp{price[0]}erp{price[1]}/'
                            redis_conn.rpush("lianjia_task_queue", task_url)
                            with counter_lock:
                                counter += 1
                            print(f"{county_name} -- {price} -- {task_url}")


# 主函数，用于启动线程
if __name__ == '__main__':
    redis_conn = redis.Redis(host='localhost', port=6379, decode_responses=True, db=0)

    url = 'https://hz.lianjia.com/zufang/'
    response = requests_get_with_retry(url)
    html_str = response.content.decode()
    root = etree.HTML(html_str)
    area_list = root.xpath("//div[@id='filter']/ul[2]/li[@class='filter__item--level2  ']/a")

    threads = []
    for area in area_list:
        area_name = "".join(area.xpath("./text()"))
        area_url = "https://hz.lianjia.com" + "".join(area.xpath("./@href"))

        # 为每个区域创建并启动一个线程
        t = Thread(target=task, args=(area_name, area_url))
        t.start()
        threads.append(t)

    # 等待所有线程完成
    for t in threads:
        t.join()

    print("所有任务已完成。")
